
/*
 * Copyright (C) Igor Sysoev
 * Copyright (C) Nginx, Inc.
 */


#ifndef _NGX_HTTP_UPSTREAM_H_INCLUDED_
#define _NGX_HTTP_UPSTREAM_H_INCLUDED_


#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_event.h>
#include <ngx_event_connect.h>
#include <ngx_event_pipe.h>
#include <ngx_http.h>


#define NGX_HTTP_UPSTREAM_FT_ERROR           0x00000002
#define NGX_HTTP_UPSTREAM_FT_TIMEOUT         0x00000004
#define NGX_HTTP_UPSTREAM_FT_INVALID_HEADER  0x00000008
#define NGX_HTTP_UPSTREAM_FT_HTTP_500        0x00000010
#define NGX_HTTP_UPSTREAM_FT_HTTP_502        0x00000020
#define NGX_HTTP_UPSTREAM_FT_HTTP_503        0x00000040
#define NGX_HTTP_UPSTREAM_FT_HTTP_504        0x00000080
#define NGX_HTTP_UPSTREAM_FT_HTTP_403        0x00000100
#define NGX_HTTP_UPSTREAM_FT_HTTP_404        0x00000200
#define NGX_HTTP_UPSTREAM_FT_UPDATING        0x00000400
#define NGX_HTTP_UPSTREAM_FT_BUSY_LOCK       0x00000800
#define NGX_HTTP_UPSTREAM_FT_MAX_WAITING     0x00001000
#define NGX_HTTP_UPSTREAM_FT_NOLIVE          0x40000000
#define NGX_HTTP_UPSTREAM_FT_OFF             0x80000000

#define NGX_HTTP_UPSTREAM_FT_STATUS          (NGX_HTTP_UPSTREAM_FT_HTTP_500  \
                                             |NGX_HTTP_UPSTREAM_FT_HTTP_502  \
                                             |NGX_HTTP_UPSTREAM_FT_HTTP_503  \
                                             |NGX_HTTP_UPSTREAM_FT_HTTP_504  \
                                             |NGX_HTTP_UPSTREAM_FT_HTTP_403  \
                                             |NGX_HTTP_UPSTREAM_FT_HTTP_404)

#define NGX_HTTP_UPSTREAM_INVALID_HEADER     40


#define NGX_HTTP_UPSTREAM_IGN_XA_REDIRECT    0x00000002 //fzc: OR到ngx_http_upstream_conf_t->ignore_headers即使upstream机制忽略响应中的对应头部
#define NGX_HTTP_UPSTREAM_IGN_XA_EXPIRES     0x00000004
#define NGX_HTTP_UPSTREAM_IGN_EXPIRES        0x00000008
#define NGX_HTTP_UPSTREAM_IGN_CACHE_CONTROL  0x00000010
#define NGX_HTTP_UPSTREAM_IGN_SET_COOKIE     0x00000020
#define NGX_HTTP_UPSTREAM_IGN_XA_LIMIT_RATE  0x00000040
#define NGX_HTTP_UPSTREAM_IGN_XA_BUFFERING   0x00000080
#define NGX_HTTP_UPSTREAM_IGN_XA_CHARSET     0x00000100


typedef struct {
    ngx_msec_t                       bl_time;
    ngx_uint_t                       bl_state;

    ngx_uint_t                       status; //fzc: 当前的记录的状态
    time_t                           response_sec; //fzc: 当前记录的状态所耗的时间, 单位是秒和毫秒
    ngx_uint_t                       response_msec;
    off_t                            response_length; //fzc: 接收到的上游响应包体长度

    ngx_str_t                       *peer;
} ngx_http_upstream_state_t; //fzc: r->upstream_states是ngx_http_upstream_state_t数组


typedef struct {
    ngx_hash_t                       headers_in_hash;
    ngx_array_t                      upstreams;
                                             /* ngx_http_upstream_srv_conf_t */
} ngx_http_upstream_main_conf_t;

typedef struct ngx_http_upstream_srv_conf_s  ngx_http_upstream_srv_conf_t;

typedef ngx_int_t (*ngx_http_upstream_init_pt)(ngx_conf_t *cf,
    ngx_http_upstream_srv_conf_t *us);
typedef ngx_int_t (*ngx_http_upstream_init_peer_pt)(ngx_http_request_t *r,
    ngx_http_upstream_srv_conf_t *us);


typedef struct {
    ngx_http_upstream_init_pt        init_upstream; //fzc: 上游主机初始化函数init_upstream被调用后设置了上游peer初始化函数init及其data, 还有包含其的ngx_http_upstream_srv_conf_s结构中的flags和servers, 如果用户没有设置, 那么就默认为ngx_http_upstream_init_round_robin
    ngx_http_upstream_init_peer_pt   init;
    void                            *data; //fzc: 如果是round robin算法, 那么存放的是ngx_http_upstream_rr_peers_s, 见ngx_http_upstream_init_round_robin
} ngx_http_upstream_peer_t;


typedef struct {
    ngx_addr_t                      *addrs;
    ngx_uint_t                       naddrs;
    ngx_uint_t                       weight;
    ngx_uint_t                       max_fails;
    time_t                           fail_timeout;

    unsigned                         down:1;
    unsigned                         backup:1;
} ngx_http_upstream_server_t;


#define NGX_HTTP_UPSTREAM_CREATE        0x0001 //fzc: 对应server指令
#define NGX_HTTP_UPSTREAM_WEIGHT        0x0002 //fzc: 让server指令获得weight
#define NGX_HTTP_UPSTREAM_MAX_FAILS     0x0004 //fzc: 允许max_fails选项
#define NGX_HTTP_UPSTREAM_FAIL_TIMEOUT  0x0008 //fzc: 允许fail_timeout选项
#define NGX_HTTP_UPSTREAM_DOWN          0x0010
#define NGX_HTTP_UPSTREAM_BACKUP        0x0020


struct ngx_http_upstream_srv_conf_s {
    ngx_http_upstream_peer_t         peer; //fzc: 一个upstream srv配置中所有peer相关的方法和数据
    void                           **srv_conf;

    ngx_array_t                     *servers;  /* ngx_http_upstream_server_t */ //fzc: server单位, server相关的数据

    ngx_uint_t                       flags;
    ngx_str_t                        host; //fzc: host即upstream指令后面的参数
    u_char                          *file_name;
    ngx_uint_t                       line;
    in_port_t                        port;
    in_port_t                        default_port;
    ngx_uint_t                       no_port;  /* unsigned no_port:1 */
};


typedef struct {
    ngx_addr_t                      *addr;
    ngx_http_complex_value_t        *value;
} ngx_http_upstream_local_t;


typedef struct {
    ngx_http_upstream_srv_conf_t    *upstream; //fzc: 未赋值ngx_http_upstream_s->resolved成员时, upstream这个结构体才会生效, 它会定义上游服务器的配置

    ngx_msec_t                       connect_timeout; //fzc: 连接上即可写事件发生的超时时间
    ngx_msec_t                       send_timeout; //fzc:? 同上
    ngx_msec_t                       read_timeout; //fzc: 可读事件发生的超时时间
    ngx_msec_t                       timeout;

    size_t                           send_lowat; //fzc: socket级别的SO_SNDLOWAT选项
    size_t                           buffer_size; //fzc: 定义接收头部的缓冲区大小(ngx_http_upstream_s->buffer), 当前不转发响应或转发响应但buffering为0时也表示接收包体的缓冲区大小

    size_t                           busy_buffers_size; //fzc: 当向下游转发响应且buffering标志位为1时生效, 它会设置到ngx_event_pipe_t->busy_size成员. 
    size_t                           max_temp_file_size; //fzc: 当buffering为1时, 如果上游速度快于下游速度, 将有可能把来自上游的响应存储到临时文件中, 而max_temp_file_size指定了临时文件的最大长度, 实际上, 他将限制ngx_event_pipe_t结构体中的temp_file.
    size_t                           temp_file_write_size; //fzc: 定义将缓冲区中的响应写入到临时文件时一次写入字符流的最大长度 

    size_t                           busy_buffers_size_conf;
    size_t                           max_temp_file_size_conf;
    size_t                           temp_file_write_size_conf;

    ngx_bufs_t                       bufs; //fzc: 转发上游响应且buffering为1时使用, 可分配的内存块的大小和个数, 而在buffering为0时不使用该字段(而是使用u->conf->buffer_size大小来分配u->buffer)

    ngx_uint_t                       ignore_headers; //fzc: bit标识字段, 32bit总共可以忽略32种头部信息, 但nginx目前只定义了8种, 跳转至NGX_HTTP_UPSTREAM_IGN_XA_REDIRECT即可见.
    ngx_uint_t                       next_upstream; //fzc: 用二进制来表示一些错误码, 如果处理上游响应时发现这些错误码, 那么在没有将响应转发到下游客户端时, 将会选择下一个上游服务器来重发请求.
    ngx_uint_t                       store_access; //fzc: 在buffering为1且转发响应时, 则有可能将响应存放到临时文件里, 在ngx_http_upstream_t->store标志位为1时, store_access表示所创建的目录, 文件的权限.
    ngx_flag_t                       buffering; //fzc: 决定转发响应方法的标志位, 为1时表示打开缓存, 尽量在内存或磁盘上缓存响应, 为0时则只开辟一片固定大小的内存来缓存响应
    ngx_flag_t                       pass_request_headers;
    ngx_flag_t                       pass_request_body;

    ngx_flag_t                       ignore_client_abort; //fzc: 为1时表示与上游服务器交互时不会检查与下游客户端之间的连接是否断开, 也就是说即使下游客户端主动关闭了连接, 也不会中断与上游服务器间的交互.
    ngx_flag_t                       intercept_errors; //fzc: 当解析到上游响应的包头headers_in结构体中的status_n错误码大于400, 则会试图把它与error_page中指定的错误码相匹配, 如果匹配上, 则发送error_page中指定的响应, 否则继续返回上游服务器的错误码. 详见ngx_http_upstream_intercept_errors方法
    ngx_flag_t                       cyclic_temp_file; //fzc: 当buffering为1且转发响应时才有意义, 为1时表示试图复用临时文件中已经使用过的空间.

    ngx_path_t                      *temp_path; //fzc: 在buffering为1且转发响应时, 存放临时文件的路径.

    ngx_hash_t                       hide_headers_hash; //fzc: 不转发的头部, 实际上是通过ngx_http_upstream_hide_headers_hash方法, 根据hide_headers和pass_headers动态数组构造出的需要隐藏的http头部散列表.
    ngx_array_t                     *hide_headers;
    ngx_array_t                     *pass_headers;

    ngx_http_upstream_local_t       *local; //fzc: 连接上服务器时使用的本机地址.

#if (NGX_HTTP_CACHE)
    ngx_shm_zone_t                  *cache;

    ngx_uint_t                       cache_min_uses;
    ngx_uint_t                       cache_use_stale;
    ngx_uint_t                       cache_methods;

    ngx_flag_t                       cache_lock;
    ngx_msec_t                       cache_lock_timeout;

    ngx_flag_t                       cache_revalidate;

    ngx_array_t                     *cache_valid;
    ngx_array_t                     *cache_bypass;
    ngx_array_t                     *no_cache;
#endif

    ngx_array_t                     *store_lengths; //fzc:? 和temp_path什么区别
    ngx_array_t                     *store_values;

    signed                           store:2; //fzc:? 目前为止, 与ngx_http_upstream_s->store相同, 为何多此一举
    unsigned                         intercept_404:1; //fzc: 针对上面的intercept_errors, 当为1时则如果上游返回404, 则不会去error_page比较, 直接转发上游404响应给下游
    unsigned                         change_buffering:1; //fzc: 为1时则表示可以根据上游响应的headers_in中的X-Accel-Buffering头部来决定buffering的值, 为Yes则buffering为1, 即可以动态的决定是以上游网速优先还是以下游网速优先.

#if (NGX_HTTP_SSL)
    ngx_ssl_t                       *ssl;
    ngx_flag_t                       ssl_session_reuse;
#endif

    ngx_str_t                        module; //fzc: 使用upstream的模块名称, 仅用于打印日志
} ngx_http_upstream_conf_t;


typedef struct {
    ngx_str_t                        name;
    ngx_http_header_handler_pt       handler;
    ngx_uint_t                       offset;
    ngx_http_header_handler_pt       copy_handler;
    ngx_uint_t                       conf;
    ngx_uint_t                       redirect;  /* unsigned   redirect:1; */
} ngx_http_upstream_header_t;


typedef struct {
    ngx_list_t                       headers;

    ngx_uint_t                       status_n;
    ngx_str_t                        status_line;

    ngx_table_elt_t                 *status;
    ngx_table_elt_t                 *date;
    ngx_table_elt_t                 *server;
    ngx_table_elt_t                 *connection;

    ngx_table_elt_t                 *expires;
    ngx_table_elt_t                 *etag;
    ngx_table_elt_t                 *x_accel_expires;
    ngx_table_elt_t                 *x_accel_redirect;
    ngx_table_elt_t                 *x_accel_limit_rate;

    ngx_table_elt_t                 *content_type;
    ngx_table_elt_t                 *content_length;

    ngx_table_elt_t                 *last_modified;
    ngx_table_elt_t                 *location;
    ngx_table_elt_t                 *accept_ranges;
    ngx_table_elt_t                 *www_authenticate;
    ngx_table_elt_t                 *transfer_encoding;

#if (NGX_HTTP_GZIP)
    ngx_table_elt_t                 *content_encoding;
#endif

    off_t                            content_length_n;

    ngx_array_t                      cache_control;

    unsigned                         connection_close:1;
    unsigned                         chunked:1;
} ngx_http_upstream_headers_in_t;


typedef struct {
    ngx_str_t                        host;
    in_port_t                        port;
    ngx_uint_t                       no_port; /* unsigned no_port:1 */

    ngx_uint_t                       naddrs; //fzc: 地址个数
    ngx_addr_t                      *addrs;

    struct sockaddr                 *sockaddr; //fzc:上游服务器地址
    socklen_t                        socklen;

    ngx_resolver_ctx_t              *ctx;
} ngx_http_upstream_resolved_t;


typedef void (*ngx_http_upstream_handler_pt)(ngx_http_request_t *r,
    ngx_http_upstream_t *u);


struct ngx_http_upstream_s {
    ngx_http_upstream_handler_pt     read_event_handler; //fzc: 与upstream的连接上的读写发生时调用的读写回调
    ngx_http_upstream_handler_pt     write_event_handler;

    ngx_peer_connection_t            peer; //fzc: 表示主动向upstream发起的连接结构体 

    ngx_event_pipe_t                *pipe; //fzc: ngx_event_pipe_t是用在转发包体且buffering为1时的核心结构体, 任何使用该方法转发响应的http模块都需要实例化该字段. 见ngx_http_proxy_handler

    ngx_chain_t                     *request_bufs; //fzc: 保存发送给upstream的请求数据, 一般直接使用从上游接收到的请求包体r->request_body->bufs, 见ngx_http_upstream_init_request

    ngx_output_chain_ctx_t           output; //fzc: 定义了向下游发送响应的方式
    ngx_chain_writer_ctx_t           writer;

    ngx_http_upstream_conf_t        *conf; //fzc: upstream访问时的所有限制性参数, 必须设置, 否则nginx会crash, 一般使用upstream机制的http模块的loc配置里都会upstream_conf结构体, 然后在其handler中将其upstream_conf赋值给r->upstream->conf, 即各个不同的http模块都是用upstream机制, 其特性都通过指令配置到upstream_conf里, 然后传递给r->upstream

    ngx_http_upstream_headers_in_t   headers_in; //fzc: upstream解析上游响应, 解析出的头部适配为http头部, 存放在该字段中

    ngx_http_upstream_resolved_t    *resolved; //fzc: 通过resolved可以直接指定上游服务器地址

    ngx_buf_t                        from_client;

    ngx_buf_t                        buffer; //fzc: 用于存储来自上游服务器的响应内容, 会被多次复用
    off_t                            length; //fzc: 表示来自上游服务器的响应包体的剩余长度, content-length字段值减去已接受的长度

    ngx_chain_t                     *out_bufs;
    ngx_chain_t                     *busy_bufs; //fzc: 在bufferring字段为0时机下游网速优先时, 存放上次转发响应未发送完的数据
    ngx_chain_t                     *free_bufs; //fzc: 同样是在buffering字段为0时, 回收out_bufs中已经发送给下游的ngx_buf_t结构体

    ngx_int_t                      (*input_filter_init)(void *data); //fzc: 处理包体前的初始化方法, 其中data参数实际就是下面的input_filter_ctx指针
    ngx_int_t                      (*input_filter)(void *data, ssize_t bytes); //fzc: 处理包体的方法, bytes表示本次接收到的包体长度
    void                            *input_filter_ctx;

#if (NGX_HTTP_CACHE)
    ngx_int_t                      (*create_key)(ngx_http_request_t *r);
#endif
    ngx_int_t                      (*create_request)(ngx_http_request_t *r); //fzc: 以下几个都是在http模块的handler中设置的, 构造发往上游服务器的请求
    ngx_int_t                      (*reinit_request)(ngx_http_request_t *r); //fzc: 与上游服务器的通信失败, 如果按照重试规则还需要再次向上游服务器发起连接时则被调用.
    ngx_int_t                      (*process_header)(ngx_http_request_t *r); //fzc: 解析上游返回的响应的包头, 返回NGX_AGAIN表示还没有接受完整, 返回NGX_OK表示已接受到完整的包头
    void                           (*abort_request)(ngx_http_request_t *r); //fzc: 目前nginx的模块基本都没有用到这个回调, 有定义也仅仅还是打印下日志
    void                           (*finalize_request)(ngx_http_request_t *r, //fzc: 销毁upstream请求时被调用
                                         ngx_int_t rc);
    ngx_int_t                      (*rewrite_redirect)(ngx_http_request_t *r, //fzc: 当process_header解析头部时发现Location或Refresh头部时表示重定向,会继而调用该回调
                                         ngx_table_elt_t *h, size_t prefix);
    ngx_int_t                      (*rewrite_cookie)(ngx_http_request_t *r,
                                         ngx_table_elt_t *h);

    ngx_msec_t                       timeout;

    ngx_http_upstream_state_t       *state; //fzc: 主要保存与上游交互的状态码和响应长度

    ngx_str_t                        method; //fzc: 不使用文件缓存时没有意义
    ngx_str_t                        schema; //fzc: 以下两个仅在记录日志被用到
    ngx_str_t                        uri;

    ngx_http_cleanup_pt             *cleanup;

    unsigned                         store:1; //fzc: 为1时表示指定文件缓存路径
    unsigned                         cacheable:1; //fzc: 为1时表示启动文件缓存
    unsigned                         accel:1; //fzc:? 未知, 加速标识? 表示解析X-Accel-*头部?
    unsigned                         ssl:1; //fzc: 为1时表示基于SSL协议访问上游服务器
#if (NGX_HTTP_CACHE)
    unsigned                         cache_status:3;
#endif

    unsigned                         buffering:1; //fzc: 当转发上游到下游的响应时, buffering为1表示开启更大的内存空间及使用临时文件来缓存来不及发送到下游的响应包体, 由于nginx和上游一般都在内网, 速度较快, 主要看nginx到下游速度的情况.
    unsigned                         keepalive:1;
    unsigned                         upgrade:1;

    unsigned                         request_sent:1; //fzc: 为1时表示是否已经向上游发送了全部请求或部分请求, 主要给ngx_output_chain用
    unsigned                         header_sent:1; //fzc: 为1时表示已经把包头转发给客户端
};


typedef struct {
    ngx_uint_t                      status;
    ngx_uint_t                      mask;
} ngx_http_upstream_next_t;


typedef struct {
    ngx_str_t   key;
    ngx_str_t   value;
    ngx_uint_t  skip_empty;
} ngx_http_upstream_param_t;


ngx_int_t ngx_http_upstream_header_variable(ngx_http_request_t *r,
    ngx_http_variable_value_t *v, uintptr_t data);

ngx_int_t ngx_http_upstream_create(ngx_http_request_t *r);
void ngx_http_upstream_init(ngx_http_request_t *r);
ngx_http_upstream_srv_conf_t *ngx_http_upstream_add(ngx_conf_t *cf,
    ngx_url_t *u, ngx_uint_t flags);
char *ngx_http_upstream_bind_set_slot(ngx_conf_t *cf, ngx_command_t *cmd,
    void *conf);
char *ngx_http_upstream_param_set_slot(ngx_conf_t *cf, ngx_command_t *cmd,
    void *conf);
ngx_int_t ngx_http_upstream_hide_headers_hash(ngx_conf_t *cf,
    ngx_http_upstream_conf_t *conf, ngx_http_upstream_conf_t *prev,
    ngx_str_t *default_hide_headers, ngx_hash_init_t *hash);


#define ngx_http_conf_upstream_srv_conf(uscf, module)                         \
    uscf->srv_conf[module.ctx_index]


extern ngx_module_t        ngx_http_upstream_module;
extern ngx_conf_bitmask_t  ngx_http_upstream_cache_method_mask[];
extern ngx_conf_bitmask_t  ngx_http_upstream_ignore_headers_masks[];


#endif /* _NGX_HTTP_UPSTREAM_H_INCLUDED_ */
